import io
import gzip
from bs4 import BeautifulSoup
from http.cookiejar import MozillaCookieJar
from urllib.request import HTTPCookieProcessor, build_opener
from urllib.request import Request
from config import config


def get_cookie(url):
    cookie = MozillaCookieJar(config.COOKIE_FILE)

    #声明一个CookieJar对象实例来保存cookie
    handle = HTTPCookieProcessor(cookie)
    # 通过handle来构建opener
    opener = build_opener(handle)
    # 此open方法与urllib2的urlopen方法，可以传入request
    header = {
        'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
        'accept': 'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'accept-encoding': 'gzip, deflate',
        'accept-language': 'en-US,en;q=0.5',
        'connection': 'keep-alive',
        'upgrade-insecure-requests': 1
    }
    req = Request(url, None, headers=header)
    response = opener.open(req)

    cookie.save(ignore_discard=True, ignore_expires=True)


def html_code(url_data):
    bi = io.BytesIO(url_data)
    gf = gzip.GzipFile(fileobj=bi, mode='rb')
    html = gf.read().decode('utf8', 'ignore')

    return BeautifulSoup(html)


def get_url_info(url_main, file):
    header = {
        'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
        'host': 'www.cbrc.gov.cn',
        'accept': 'ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'accept-encoding': 'gzip, deflate',
        'accept-language': 'en-US,en;q=0.5',
        'connection': 'keep-alive',
        'upgrade-insecure-requests': 1
    }
    url = '{}{}'.format(url_main, file)

    cookie = MozillaCookieJar()
    cookie.load(config.COOKIE_FILE, ignore_discard=True, ignore_expires=True)

    req = Request(url, None, headers=header)
    opener = build_opener(HTTPCookieProcessor(cookie))

    data = opener.open(req).read()
    return html_code(url_data=data)
